#!/usr/bin/env python
# -*- coding:utf-8 -*-
import sys
reload(sys)
sys.setdefaultencoding('utf-8')  # for BeautifulSoup ?
import BeautifulSoup
import locale
import re
import urlparse
import types
import pprint
import TrainGrade
import PageOpener
PASS_STRING = u"レ"
REPEAT_STRING = u"〃"
TRAIN_HEADERS = {
    u"列車名":         'name',
    u"列車番号":       'number',
    u"列車予約コード": 'resaave',
    u"連結車両":       'car',
    u"備考":           'note',
    u"運転日":         'date',
}
rePass = re.compile(u"レ")
#reDeparture = re.compile(u"([0-9][0-9]:[0-9][0-9])発", re.U)
reDeparture = re.compile(u"([0-9][0-9]:[0-9][0-9])\)?発", re.U)  # 臨時対応 (HH:MM)発
#reArrive = re.compile(u"([0-9][0-9]:[0-9][0-9])着", re.U)
reArrive = re.compile(u"([0-9][0-9]:[0-9][0-9])\)?着", re.U)  # 臨時対応 (HH:MM)着
rePage = re.compile(u"[0-9]+頁", re.U)
reHeader = re.compile(u"\[(\S+)\u301c(\S+)\]\uff1a\s*(\S+)", re.U)
#reType = re.compile(u"^(.*)(普通|急行|快速|特急|準急)(.*)", re.U) → TrainGrade.reType()
reGou = re.compile(u"(\D*)(\d+)号", re.U)


class ParseEkikara:
    def __init__(self,
                 logger,
                 status_option,
                 console_encode=locale.getpreferredencoding()):
        self.logger = logger       # logging object
        self.console_encode = console_encode
        self.status_option = status_option
        self.pp = pprint.PrettyPrinter(indent=2)
        self.grade = TrainGrade.TrainGrade(logger)
        # for getTimeTableAttr()
        # ディズニーリゾートライン&nbsp;リゾートゲートウェイ方面&nbsp;休日&nbsp;1/14頁
        self.reNowPage = re.compile(u"\d+/\d+頁")
        self.reTitle = re.compile(u"\[(\S+)\](\S+)&nbsp;(\S+)方面[(（](\S+)[)）]&nbsp;(\S+)&nbsp;\d+/\d+頁")
        # [神戸新交通]ポートアイランド線&nbsp;三宮方面(北埠頭経由)(上り)&nbsp;平日&nbsp;1/10頁
        self.reTitleVia = re.compile(u"\[(\S+)\](\S+)&nbsp;(\S+)方面[(（](\S+)[)）][(（](\S+)[)）]&nbsp;(\S+)&nbsp;\d+/\d+頁")
        self.reTitleNoUpdown = re.compile(u"\[(\S+)\](\S+)&nbsp;(\S+)方面&nbsp;(\S+)&nbsp;\d+/\d+頁")
        self.reSmallTitle = re.compile(u"(\S+)&nbsp;(\S+)方面[(（](\S+)[)）]&nbsp;(\S+)&nbsp;\d+/\d+頁")
        self.reSmallTitleNoUpdown = re.compile(u"(\S+)&nbsp;(\S+)方面&nbsp;(\S+)&nbsp;\d+/\d+頁")

    def _statusMsg(self, s):
        sys.stderr.write(s.encode(self.console_encode))

    def getHeaderValues(self, tdBlock):
        values = []
        for item in tdBlock.contents:
            if item.string:
                m = reHeader.search(item.string.strip())
                if m:
                    value = {}
                    value['from'] = m.group(1)
                    value['to'] = m.group(2)
                    value['value'] = m.group(3)
                    values.append(value)
        return values

    def getHeaderText(self, tdBlock):
        values = []
        for item in tdBlock.contents:
            if item.string:
                values.append(item.string.strip())
        return values

    def infoStations(self, stations):
        u""" stations return for log str.
        """
        msgs = []
        for station in stations:
            msg = station['name']
            if 'arrival' in station and station['arrival']:
                msg += u"a"
            if 'departure' in station and station['departure']:
                msg += u"d"
            msgs.append(msg)
        return msgs

    def getStationList(self, pageOpener, url, updown='down'):
        u""" get station list from timetable page.

        2011/01/08 by H.Haga 路線の分岐に対応(但しリストは分岐を考慮せずのべたんで出力)
        2011/03/31 by H.Haga 環状線等で同じ駅名があった場合でも頓着せずにそのまま出力するようにする
        2011/08/10 by H.Haga 上り時刻表から得られた駅名リストは反転して返すようにした。
        """
        stations = []
        nameMap = []       # 直近の駅名の index 位置
        index = 0
        html = pageOpener.open(fullurl=url)
        soup = BeautifulSoup.BeautifulSoup(html)
        # get station list
        startPoint = soup.find(lambda tag:
                                   tag.name == 'a' and 'station' in tag.get('href'))
        del soup
        if startPoint != None:
            stationBlock = startPoint.parent.parent.parent.parent      # aタグを含む tr タグを取得
            while stationBlock:
                item = stationBlock.find(lambda tag:
                                             tag.name == 'a' and 'station' in tag.get('href'))
                if item != None:
                    while item != None:
                        station = {}
                        if item.__class__.__name__ == 'Tag' and item.name == u'a':
                            station['name'] = item.string
                            self.logger.debug(u"[%s]" % (station['name'],))
                            stations.append(station)
                            nameMap.append(len(stations) - 1)
                        elif item.__class__.__name__ == 'NavigableString' and item.string.strip() == REPEAT_STRING:
                            nameMap.append(len(stations) - 1)
                        item = item.nextSibling
                    aBlock = stationBlock.find(lambda tag:
                                                   tag.name == 'a' and 'ekijikoku' in tag.get('href'))
                    if aBlock != None:
                        element = aBlock.parent.parent.next
                        if element != None:
                            while element != None:
                                if element.__class__.__name__ == 'Tag' and element.name == 'span':
                                    if element.string:
                                        stations[nameMap[index]]['arrival'] = True
                                        self.logger.debug(u"staitons[%d]=[%s] arrival true" %
                                                          (nameMap[index],
                                                           stations[nameMap[index]]['name']))
                                        index += 1
                                    else:
                                        stations[nameMap[index]]['departure'] = True
                                        self.logger.debug(u"staitons[%d]=[%s] departure true" %
                                                          (nameMap[index],
                                                           stations[nameMap[index]]['name']))
                                        index += 1
                                element = element.findNextSibling('span')
                if stationBlock != None:
                    stationBlock = stationBlock.findNextSibling('tr')     # 次のtrタグに駅名が無いか調査するため。
        if updown == 'up':      # 上り時刻表から得られた駅名リストは反転する。
            stations.reverse()
        msgStations = u"parsed stations:" + (u",".join(self.infoStations(stations)))
        self.logger.debug(msgStations)
        if self.status_option:
            self._statusMsg(msgStations + u"\n")
        return stations

    def getTimeTableAttr(self, pageOpener, url):
        attr = {}
        html = pageOpener.open(fullurl=url)
        soup = BeautifulSoup.BeautifulSoup(html)
        spans = soup.findAll(lambda tag: tag.name == 'span' and (tag.get('class') and 'textBold' in tag.get('class')))
        del soup
        if spans:
            for span in spans:
                #[名古屋市営]名城線&nbsp;ナゴヤドーム前矢田方面（左回り）&nbsp;平日&nbsp;1/16頁
                #[神戸新交通]ポートアイランド線&nbsp;三宮方面(北埠頭経由)(上り)&nbsp;平日&nbsp;1/10頁
                if span.string and self.reNowPage.search(span.string.strip()):
                    mTitleVia = self.reTitleVia.search(span.string.strip())
                    if mTitleVia:      # reTitleにもマッチしてしまうのでreTitleViaを先にチェック
                        attr['company'] = mTitleVia.group(1)
                        attr['line'] = mTitleVia.group(2)
                        attr['direction'] = mTitleVia.group(3) + mTitleVia.group(4)
                        attr['updown'] = mTitleVia.group(5)
                        attr['day'] = mTitleVia.group(6)
                    else:
                        mTitle = self.reTitle.search(span.string.strip())
                        if mTitle:
                            attr['company'] = mTitle.group(1)
                            attr['line'] = mTitle.group(2)
                            attr['direction'] = mTitle.group(3)
                            attr['updown'] = mTitle.group(4)
                            attr['day'] = mTitle.group(5)
                        else:
                            mSmallTitle = self.reSmallTitle.search(span.string.strip())
                            if mSmallTitle:
                                attr['company'] = mSmallTitle.group(1)
                                attr['line'] = u""
                                attr['direction'] = mSmallTitle.group(2)
                                attr['updown'] = mSmallTitle.group(3)
                                attr['day'] = mSmallTitle.group(4)
                            else:
                                # 2011/01/23 by kuma35
                                # 方面の後に(上り)等のカッコ無しパターン.
                                # 例:[伊予鉄道]３系統(市駅線) 道後温泉方面 平日 1/9頁
                                mTitleNoUpdown = self.reTitleNoUpdown.search(span.string.strip())
                                if mTitleNoUpdown:
                                    attr['company'] = mTitleNoUpdown.group(1)
                                    attr['line'] = mTitleNoUpdown.group(2)
                                    attr['direction'] = mTitleNoUpdown.group(3)
                                    attr['updown'] = attr['direction']      # updownにdirectionをそのままセットしておく
                                    attr['day'] = mTitleNoUpdown.group(4)
                                else:
                                    # 2011/08/06 for ディズニーリゾートライン。
                                    mSmallTitleNoUpdown = self.reSmallTitleNoUpdown.search(span.string.strip())
                                    if mSmallTitleNoUpdown:
                                        attr['company'] = mSmallTitleNoUpdown.group(1)
                                        attr['line'] = u""
                                        attr['direction'] = mSmallTitleNoUpdown.group(2)
                                        attr['updown'] = attr['direction']      # updownにdirectionをそのままセットしておく
                                        attr['day'] = mSmallTitleNoUpdown.group(3)
        return attr

    def getPages(self, pageOpener, url):
        pages = []
        html = pageOpener.open(fullurl=url)
        soup = BeautifulSoup.BeautifulSoup(html)
        # own page and other pages
        for optionBlock in soup.findAll(lambda tag: tag.name == 'option'):
            if optionBlock.string:
                if rePage.search(optionBlock.string):
                    pageUrl = urlparse.urljoin(url, optionBlock.get('value') + ".htm")
                    self.logger.debug(u"pageUrl=[%s]" % (pageUrl,))
                    pages.append(pageUrl)
        self.logger.debug(self.pp.pformat(pages))
        msgCount = u"page url count=[%d]" % (len(pages),)
        self.logger.debug(msgCount)
        if self.status_option:
            self._statusMsg(msgCount + u"\n")
        return pages

    def getDetails(self, pageOpener, url):
        details = []
        detailsSet = set(details)
        for page in self.getPages(pageOpener, url):
            html = pageOpener.open(fullurl=page)
            soup = BeautifulSoup.BeautifulSoup(html)
            for detailUrl in soup.findAll(lambda tag: tag.name == 'a' and
                                          'detail' in tag.get('href') and
                                          u"詳細" in tag.contents):
                detailPageUrl = urlparse.urljoin(page, detailUrl.get('href'))
                self.logger.debug(u"detailUrl=[%s]" % (detailPageUrl,))
                if not detailPageUrl in detailsSet:
                    details.append(detailPageUrl)
                    #detailSet = set(details)
            if self.status_option:
                self._statusMsg(u"[%d]" % (len(details),))
        self.logger.debug(self.pp.pformat(details))
        if self.status_option:
            self._statusMsg(u"\ndetail page url total count=[%d]\n" %
                              (len(details),))
        return details

    def getTrainHeader(self, tdBlocks):
        # number が types.ListType でなくても、他のパラメータが ListType の可能性がある hoge～fuga：.... 形式の時
        # number が正義とする(2010/09/14)
        train = {}
        strTitleColumn = None
        for titleColumn in tdBlocks[0].findAll(lambda tag: tag.name == 'span' and 'textBold' in tag.get('class')):
            if titleColumn.string:
                strTitleColumn = titleColumn.string.strip()
                self.logger.debug(u"getTrainHeader strTitleColumn=[%s]" % (strTitleColumn,))
                if strTitleColumn in TRAIN_HEADERS:
                    for titleValue in tdBlocks[1].findAll(lambda tag: tag.name == 'span' and tag.get('class') == 'l'):
                        if titleValue.string:
                            train[TRAIN_HEADERS[strTitleColumn]] = titleValue.string.strip()
                        else:
                            headerValues = self.getHeaderValues(titleValue)
                            if  headerValues:
                                train[TRAIN_HEADERS[strTitleColumn]] = headerValues
                            else:
                                # 複数行(複数タグ)を単一文字列にする。
                                train[TRAIN_HEADERS[strTitleColumn]] = "\n".join(self.getHeaderText(titleValue))
        return train

    def getTrainTime(self, tdBlocks):
        station = {}
        station_order = 0
        for stationName in tdBlocks[0].findAll(lambda tag: tag.name == 'a' and  'station' in tag.get('href')):
            station = {'name': None, 'arrival': None, 'departure': None, 'platform': None}
            if stationName.string:
                station_order += 1
                station['order'] = station_order
                station['name'] = stationName.string
                for stationTime in tdBlocks[1].findAll(lambda tag: tag.name == 'span' and tag.get('class') == 'l'):
                    if stationTime.string:
                        matchPass = rePass.search(stationTime.string.strip())
                        if  matchPass:
                            station['arrival'] = PASS_STRING
                            station['departure'] = PASS_STRING
                        else:
                            matchArrive = reArrive.search(stationTime.string.strip())
                            if matchArrive:
                                station['arrival'] = matchArrive.group(1)
                            else:
                                matchDeparture = reDeparture.search(stationTime.string.strip())
                                if matchDeparture:
                                    station['departure'] = matchDeparture.group(1)
                                else:
                                    pass
                for platform in tdBlocks[2].findAll(lambda tag: tag.name == 'span' and tag.get('class') == 'l'):
                    if platform.string:
                        if platform.string.strip() == u"&nbsp;"  or platform.string.strip() == u"":
                            station['platform'] = u""
                        else:
                            station['platform'] = platform.string.strip()
            return station

    def getTrain(self, pageOpener, url):
        train = {'stations': [], 'url': url}
        html = pageOpener.open(fullurl=url)
        soup = BeautifulSoup.BeautifulSoup(html)
        for divBlock in soup.findAll(name='div', attrs={'id': 'container02'}):
            for trBlock in divBlock.findAll('tr'):
                tdBlocks = trBlock.findAll('td')
                if  len(tdBlocks) == 2:
                    header = self.getTrainHeader(tdBlocks)
                    if header:
                        train.update(header)
                elif len(tdBlocks) == 3:
                    detail = self.getTrainTime(tdBlocks)
                    if detail:
                        train['stations'].append(detail)
        self.logger.debug(self.pp.pformat(train))
        if self.status_option and TRAIN_HEADERS[u"列車番号"] in train:
            strMsg = u"\nparsed"
            number = train[TRAIN_HEADERS[u"列車番号"]]
            name = train[TRAIN_HEADERS[u"列車名"]]
            if isinstance(number, types.StringTypes):
                strMsg += u"[%s]" % (number,)
            elif isinstance(number, types.ListType):
                strMsg += u"["
                for item in number:
                    strMsg += u"[%s]～[%s]:[%s]," % (item['from'], item['to'], item['value'],)
                strMsg += u"]"
            if isinstance(name, types.StringTypes):
                strMsg += u"[%s]" % (name,)
            elif isinstance(name, types.ListType):
                strMsg += u"["
                for item in name:
                    strMsg += u"[%s]～[%s]:[%s]," % (item['from'], item['to'], item['value'],)
                strMsg += u"]"
            self._statusMsg(strMsg)
        return train

    def splitItemToStr(self, items):
        u"""items を単一文字列にして返す

        " ".join("hoge〜fuga:tee")
        """
        resultList = []
        for item in items:
            resultList.append(u"%s〜%s：%s" % (item['from'], item['to'], item['value']))
        return u" ".join(resultList)

    def splitTrainItem(self, numbers, items):
        u"""item(s)を返す。numberと並びが違う時は同一文字列のリストを返す

        phase1.numberと違う時は、単一文字列に合成した上で、その単一文字列を値に持つリストを返す。
        (phase2.numberと違う時は、単なる省略であれば分割する。無理なときは上記)
        """
        result = []
        if isinstance(items, types.ListType):
            if len(numbers) == len(items):
                for number, item in zip(numbers, items):
                    if number['from'] == item['from'] and number['to'] == item['to']:
                        result.append({'from': item['from'], 'to': item['to'], 'value': item['value']})
                    else:
                        value = self.splitItemToStr(items)
                        result.append({'from': number['from'], 'to': number['to'], 'value': value})
            else:
                value = self.splitItemToStr(items)
                for number in numbers:
                    result.append({'from': number['from'], 'to': number['to'], 'value': value})
        else:
            for i in range(len(numbers)):
                result.append({'from': numbers[i]['from'], 'to': numbers[i]['to'], 'value': items})
        return result

    def splitTrain(self, train):
        u"""train['number']がListTypeである場合、単一の列車番号に分割する。

        Todo:最初に分割前の情報を返すようにする。
        Todo:train_headerに分割が分かる情報を持たせる
        """
        trains = []
        # number とそろっているかどうか確認し、そろってないものは
        # 0.大原則: numberが正義とする
        # (1. 一部省略であり、展開可能なものは展開する。)
        # 2. number と互換が無い場合はlistをstringにする。
        #    互換の無い場合で hoge〜fuga:tee 形式で無いものは getHeaderText にて既に文字列になっているので、
        #    ここでは hoge〜fuga:tee 形式かつnumberと互換の無いものが対象となる。
        train_name = self.splitTrainItem(train['number'], train['name'])
        train_date = self.splitTrainItem(train['number'], train['date'])
        train_note = self.splitTrainItem(train['number'], train['note'])
        # 要LOOP対応。列車番号、列車名に示された区間の特定はこれでよいのか.....無理なので最初に分割前の情報を返す。
        stationMap = {}
        for i, v in enumerate(train['stations']):
            stationMap[v['name']] = i
        for i in range(len(train['number'])):
            assert train['number'][i]['from'] in stationMap, "from station not in stations"
            assert train['number'][i]['to'] in stationMap, "to station not in stations"
            newTrain = {'number': train['number'][i]['value'],
                        'name': train_name[i]['value'],
                        'stations': train['stations'][stationMap[train['number'][i]['from']]: stationMap[train['number'][i]['to']] + 1],  # slice is [from:to+1]:-)
                        'note': train_note[i]['value'],
                        'date': train_date[i]['value'],
                        'url': train['url'],
                        }
            trains.append(newTrain)
        return trains

    def extractTrainName(self, name):
        item = {'type': None, 'type_prefix': None, 'name': None, 'gou': None}
        name = name.strip()
        mType = self.grade.reType.search(name)
        if mType:
            item['type_prefix'] = mType.group(1)
            item['type'] = mType.group(2)
            name = mType.group(3)
        mGou = reGou.search(name)
        if mGou:
            item['gou'] = mGou.group(2)
            name = mGou.group(1)
        item['name'] = name
        return item


# --------------- test
if __name__ == '__main__':
    #import BeautifulSoup
    import getTimetableLog
    #import locale
    from optparse import OptionParser
    #import pprint
    import urllib2
    #import urlparse
    optParser = OptionParser(usage="%prog [options] URL [ URL... ]", version="%prog 0.2")
    optParser.add_option("-p", "--proxy", dest="proxy", help="specify proxy if you need. less scheme name(NG; http://hogehoge.com:8080, GOOD; hogehoge.com:8080)", metavar="URL")
    #parser.add_option("-d", "--debug", action="store_true", dest="debug", help="a lot of messages for debug")
    optParser.add_option("-s", "--status", action="store_true", dest="status", help="status messages to stderr")
    (OPTIONS, args) = optParser.parse_args()
    logger = getTimetableLog.logging.getLogger("gettimetable")
    CONSOLE_ENCODE = locale.getpreferredencoding()
    pageOpener = PageOpener.PageOpener(logger=logger,
                                       proxy=OPTIONS.proxy,
                                       console_encode=CONSOLE_ENCODE)
    parser = ParseEkikara(logger=logger,
                          status_option=OPTIONS.status,
                          console_encode=CONSOLE_ENCODE)
    train_order = 0
    for url in args:
        stationList = parser.getStationList(pageOpener, url)
        for detailUrl in parser.getDetails(pageOpener, url):
            train = parser.getTrain(pageOpener, detailUrl)
            if isinstance(train['number'], types.StringTypes):
                train.update(parser.extractTrainName(train['name']))
                train_order += 1
            elif isinstance(train['number'], types.ListType):
                for singleTrain in parser.splitTrain(train):
                    singleTrain.update(parser.extractTrainName(singleTrain['name']))
                    train_order += 1
